package com.futvan.z.job;

import com.futvan.z.erp.erp_search_engines_info.erp_search_engines_info;
import com.futvan.z.framework.core.z;
import com.futvan.z.framework.util.DateUtil;
import com.futvan.z.framework.util.StringUtil;
import org.apache.http.HttpEntity;
import org.apache.http.HttpHost;
import org.apache.http.HttpResponse;
import org.apache.http.client.config.RequestConfig;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.utils.URIBuilder;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.util.EntityUtils;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import org.quartz.JobExecutionContext;
import org.quartz.JobExecutionException;
import org.quartz.StatefulJob;

import java.util.*;

public class BaiduReptileJob implements StatefulJob {
    private final static int pages = 5;//采集页数
    private final static List<String> keywords = Arrays.asList("美罗国际", "美罗生物");//搜索关键词

    @Override
    public void execute(JobExecutionContext context) throws JobExecutionException {
        //执行采集
        Run();
    }

    /**
     * 执行采集
     */
    private void Run(){
        //循环关键词集合
        for (String keyword : keywords) {
            String keyword_encode = StringUtil.UrlEncode(keyword);
            if(z.isNotNull(keyword_encode)){
                for (int i = 0; i < pages; i++) {

                    //获取返回HTML信息
                    int page_num = i+1;
                    String url = "https://www.baidu.com/s?ie=utf-8&f=8&rsv_bp=1&rsv_idx=2&tn=SE_baiduhomet1_zehq5d4k&wd="+keyword_encode;
                    if(page_num>1) {
                        url = url +"&pn="+i*10;
                    }
                    String html = http(url, null,null,0);

                    //解析HTML
                    Document doc = Jsoup.parse(html);
                    Elements list = doc.select("#content_left div");
                    for (Element el : list) {
                        String from_url = el.select("h3 a").attr("href");//URL
                        if(z.isNotNull(from_url)){
                            //保存信息
                            erp_search_engines_info info = new erp_search_engines_info();
                            info.setZid(z.newZid());
                            info.setSearch_engines_id(this.getClass().getSimpleName());//搜索引擎
                            info.setCreate_time(DateUtil.getDateTime());//采集时间
                            info.setInfo(keyword);//关键词
                            info.setSseq(String.valueOf(page_num));//设置页号
                            info.setUrl(from_url);//URL
                            info.setTitle(el.select("h3").text().replace("...",""));//标题

                            //保存到数据库
                            Save(info);
                        }
                    }
                }
            }
        }
    }

    /**
     * 保存采集信息
     * @param info
     * @return
     */
    private boolean Save(erp_search_engines_info info) {
        boolean issave = false;
        try {
            List<HashMap<String,String>> rlist = z.getSqlSession().selectList("select","select * from erp_search_engines_info where url = '"+info.getUrl()+"'");
            if(z.isNull(rlist) || rlist.size()<=0){
                z.getSqlSession().insert("erp_search_engines_info_insert",info);
            }
            issave = true;
        }catch (Exception e){
        }
        return issave;
    }


    /**
     * HTTP请求
     * @param url
     * @param parameters
     * @return
     */
    public String http(String url, Map<String,String> parameters, String ip, int port){
        String html = "";
        try {
            CloseableHttpClient httpclient = null;
            if(z.isNotNull(ip) && port>0) {
                HttpHost proxy = new HttpHost(ip, port, "http");
                RequestConfig defaultRequestConfig = RequestConfig.custom()
                        .setConnectTimeout(20000)
                        .setSocketTimeout(20000)
                        .setConnectionRequestTimeout(20000)
                        .setProxy(proxy)
                        .build();

                //创建连接
                httpclient = HttpClients.custom().setDefaultRequestConfig(defaultRequestConfig).build();
            }else {
                RequestConfig defaultRequestConfig = RequestConfig.custom()
                        .setConnectTimeout(20000)
                        .setSocketTimeout(20000)
                        .setConnectionRequestTimeout(20000)
                        .build();

                //创建连接
                httpclient = HttpClients.custom().setDefaultRequestConfig(defaultRequestConfig).build();
            }

            //封闭参数
            URIBuilder builder = new URIBuilder(url);
            if(parameters!=null && parameters.size()>0) {
                for (Map.Entry<String, String> entry : parameters.entrySet()) {
                    builder.setParameter(entry.getKey(), entry.getValue());
                }
            }

            //创建请求
            HttpGet doGet = new HttpGet(builder.build());

            //User-Agent 为模拟chrome PC浏览器
            doGet.addHeader("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/117.0.0.0 Safari/537.36 Edg/117.0.2045.31");
            doGet.addHeader("Accept-Language", "zh-CN,zh;q=0.9,en;q=0.8,en-GB;q=0.7,en-US;q=0.6");
            doGet.addHeader("Cookie","BIDUPSID=460DA51280CBB6E8560A1C24827BA46B; PSTM=1689232011; BD_UPN=12314753; BDUSS=0tPMkYtZ2ZMfktpZXpZcDVGVUFRNWNwZ0tyMVpLVENOaEYtRjhDTERxeE1IdGhrRVFBQUFBJCQAAAAAAAAAAAEAAACjwuwBZnV0dmFuAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAEyRsGRMkbBkN; BDUSS_BFESS=0tPMkYtZ2ZMfktpZXpZcDVGVUFRNWNwZ0tyMVpLVENOaEYtRjhDTERxeE1IdGhrRVFBQUFBJCQAAAAAAAAAAAEAAACjwuwBZnV0dmFuAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAEyRsGRMkbBkN; BAIDUID=460DA51280CBB6E8C7AB25E7C96F51E6:SL=0:NR=50:FG=1; newlogin=1; MCITY=-167%3A; ispeed_lsm=2; H_WISE_SIDS=234020_216846_213356_214807_110085_244727_254833_261710_236312_256419_265881_266365_265986_265615_268029_259642_269832_269905_267066_256739_270460_270313_270520_264423_270505_270548_271173_271319_265034_271271_266027_270102_271866_271939_256151_234295_234208_272284_263618_272335_253022_272830_260335_269296_272987_267596_267558_273161_273119_273232_273300_272984_273399_273390_271157_273521_272641_273198_271562_271146_273671_273705_264170_270186_273164_274077_273958_274178_269610_273918_273788_273044_272690_263749_274300_274341_274385_272561_274591_272331_188333_274768_274762_270142_274841_274843_274854_274856_274847_270158_275069_272801_275098_267807_267547_275167_274333_275148_272318_268759_275615_274975_275776_273491_275660_275006_275826_275940_275848_275970_274785_274079_276090_8000073_8000123_8000141_8000159_8000165_8000169_8000176_8000180; H_WISE_SIDS_BFESS=234020_216846_213356_214807_110085_244727_254833_261710_236312_256419_265881_266365_265986_265615_268029_259642_269832_269905_267066_256739_270460_270313_270520_264423_270505_270548_271173_271319_265034_271271_266027_270102_271866_271939_256151_234295_234208_272284_263618_272335_253022_272830_260335_269296_272987_267596_267558_273161_273119_273232_273300_272984_273399_273390_271157_273521_272641_273198_271562_271146_273671_273705_264170_270186_273164_274077_273958_274178_269610_273918_273788_273044_272690_263749_274300_274341_274385_272561_274591_272331_188333_274768_274762_270142_274841_274843_274854_274856_274847_270158_275069_272801_275098_267807_267547_275167_274333_275148_272318_268759_275615_274975_275776_273491_275660_275006_275826_275940_275848_275970_274785_274079_276090_8000073_8000123_8000141_8000159_8000165_8000169_8000176_8000180; BDORZ=B490B5EBF6F3CD402E515D22BCDA1598; BAIDUID_BFESS=460DA51280CBB6E8C7AB25E7C96F51E6:SL=0:NR=50:FG=1; channel=baidusearch; baikeVisitId=92ce8037-b87a-4bb9-ad70-dbeadadc7b7e; BA_HECTOR=2l242la481al200l2585al881igib4r1o; ZFY=J1BhPxFyV7Abu3Hzs:AY9TkvqDrNMDh:Ak:BdvtllOdSZ0:C; COOKIE_SESSION=79980_0_6_6_8_5_1_0_6_4_4_1_80004_0_8_0_1695101667_0_1695101659%7C9%23100731_16_1694762586%7C9; BDRCVFR[feWj1Vr5u3D]=I67x6TjHwwYf0; delPer=0; BD_CK_SAM=1; PSINO=2; BDRCVFR[wBw_ZYV1Tvc]=mk3SLVN4HKm; BDRCVFR[dG2JNJb_ajR]=mk3SLVN4HKm; BDRCVFR[X_XKQks0S63]=mk3SLVN4HKm; H_PS_PSSID=39320_39365_39342_39347_39407_39097_39415_39198_39346_39261_39358_39233_26350; ab_sr=1.0.1_NzYxY2E4NDAzOGFiZmRiNjk5MGM5NGQ2YzUwMGRiNmExOTZiNjFhNDJiYzYxNzYxZmYzOWFiNTY1ODZhZDRlMmU3ZjNmNDZlYjBlN2IxNWYzMmE4NzVkZTNhNjg5MWQwMTYwYzA3ZDk2MGNkZjZmOWFhODdhYWEzZmYyMGVkNGE4NDU1MTIwNGE0ODM1YTIxYzU3MzRjYzU1Yjc5MTVmYw==; sug=3; sugstore=1; ORIGIN=0; bdime=0; H_PS_645EC=24449F%2FodQcLcC7n0nN%2BVZuHyz0Ce7glp%2BDAgBVNW5JEEIE9MdE%2Bz0qlg12V1HglZ5RZ; BDSVRTM=0");


            //如果有代理IP。设置X-Forwarded-For为代理IP，隐藏真实IP
            if(z.isNotNull(ip)) {
                doGet.setHeader("X-Forwarded-For", ip);
            }

            //执行
            HttpResponse response = httpclient.execute(doGet);
            int status = response.getStatusLine().getStatusCode();
            if (status >= 200 && status < 300) {
                HttpEntity entity = response.getEntity();
                html = entity != null ? EntityUtils.toString(entity,"utf-8"):"";
            }else {
                z.Error("HTTP ERROR : " + status);
            }
        }catch (Exception e){
            z.Error("HTTP Exception : " + e.getMessage());
        }
        return html;
    }
}
